#include "../include/WebPage.h"
#include "../include/SplitToolCppjieba.h"
#include "../include/Configuration.h"
#include "../include/simhash/Simhasher.hpp"
#include <algorithm>  
#include <iostream>
#include <stdlib.h>


//this define can avoid some logs which you don't need to care about.
#define LOGGER_LEVEL LL_WARN 


using namespace simhash;

using std::set;
using std::iostream;
using std::cout;
using std::endl;


WebPage::WebPage(const string & doc, Configuration & config, SplitToolCppjieba & jieba)
:_doc(doc)
{
    
}

WebPage::~WebPage() {}


string WebPage::getDoc() const//获取文档
{
    return _doc;
}

int WebPage::getDocId()//获取文档的id
{
    //<doc><docid>id
    int preSize = 12;//起始位置 前面12个是<doc><doccid>
    int i = preSize;

    int len = 0;
    while(_doc[i] >= '0' && _doc[i] <= '9')
    {
        ++i;
        ++len;
    }
    string s0 = _doc.substr(preSize, len);//提取数字子串
    _docId = atoi(s0.c_str());
    return _docId;
}

string WebPage::getUrl()//获取URL
{
    return _docUrl;
}

string WebPage::getTitle()//获取文章标题
{
    return _docTitle;
}    

string WebPage::getContent()//获取文章内容
{
    return _docContent;
}

map<string, int>& WebPage::getWordsMap() //获取文档的词频统计 map
{
    return _wordsMap;
}

string WebPage::getSunmary()//获取文章摘要
{
    int i = 0;

    for(auto it = _wordsMap.begin(); it != _wordsMap.end(); ++it){
        i++;
        if(i < 20){
           _docSummary +=  it->first;  
        }
    }
    return _docSummary;
}

void WebPage::processDoc(const string & doc, Configuration & config, SplitToolCppjieba & Cppjieba) //对格式化文档进行处理
{
    set<string>stopList = config.getStopWordList();//获取停用词词集

    vector<string>cutSentence =  Cppjieba.cut(_doc);//将文章分词 //每个string是一个词
    //遍历停用词词集 从文章分词中逐个删除

    for(auto it = stopList.begin(); it != stopList.end(); ++it)
    {
        string stopword = *it;
        cutSentence.erase(std::remove(cutSentence.begin(), cutSentence.end(), stopword), cutSentence.end());
    }

    //统计词频写入_wordsMap中
    for(auto word : cutSentence){
        auto ret = _wordsMap.insert({word, 1});//插入单词，次数为1
        if(!ret.second)//返回值为键值对，且first为已存的键，second为bool值，当seconde为false时，则插入失败，表示已存在
        {
            ++_wordsMap[word];//对应的键值对的词频+1
        }    
    }

    //从分词中查找第一个title并拼接到
    auto title = find(cutSentence.begin(), cutSentence.end(), "title");
    if (title != cutSentence.end()){
        cout << "查找成功：" << *title <<endl;
        ++title;//跳过">"
        ++title;//标题的第一个string位置
        for(; *title != "<"; ++title){
            _docTitle += *title;  
        }
    }
    else
    {
        cout << "查找失败" <<endl;;
    }

    //从分词中查找第一个url并拼接到<
    auto url = find(cutSentence.begin(), cutSentence.end(), "url");
    if (url != cutSentence.end()){
        cout << "查找成功：" << *url << endl;
        ++url;//跳过">"
        ++url;//标题的第一个string位置
        for(; *url != "<"; ++url){
            _docUrl += *url;  
        }
    }
    else
    {
        cout << "查找失败" <<endl;;
    }

    //从分词中查找第一个content并拼接到<
    auto content = find(cutSentence.begin(), cutSentence.end(), "content");
    if (content != cutSentence.end()){
        cout << "查找成功：" << *content << endl;
        ++content;//跳过">"
        ++content;//标题的第一个string位置
        for(; *content != "<"; ++content){
            _docContent += *content;  
        }
    }
    else
    {
        cout << "查找失败" <<endl;;
    }
    //文章摘要.....

}

bool operator==(const WebPage & lhs, const WebPage & rhs)  //判断两篇文档是否相等 
{

    //固定的传入词典路径  
    Simhasher simhasher("../include/simhash/cppjieba/dict/jieba.dict.utf8", "../include/simhash/cppjieba/dict/hmm_model.utf8",
                           "../include/simhash/cppjieba/dict/idf.utf8", "../include/simhash/cppjieba/dict/stop_words.utf8");

    size_t topN = 10;//选取权重最高的10个词
    uint64_t u1 = 0, u2 = 0;
    vector<pair<string ,double> > res1, res2;
    
    string s1 = lhs.getDoc();
    simhasher.extract(s1, res1, topN);//提取关键词和权重到res
    simhasher.make(s1, topN, u1);//计算哈希值并存到u64中

    string s2 = rhs.getDoc();
    simhasher.extract(s2, res2, topN);//提取关键词和权重到res
    simhasher.make(s2, topN, u2);//计算哈希值并存到u64中

    if(s1 == s2)
    {
        return true;
    }
    else
    {
        return false;
    }
}

bool operator < (const WebPage & lhs, const WebPage & rhs) //对文档按 Docid 进行排序
{
    if(lhs._docId < rhs._docId)
    {
        return true;
    }
    else
    {
        return false;
    }
}

